#!/usr/bin/python3
# -*- coding:utf-8 -*-
# project:
# user:86175
# Author: 亿只羊
# createtime: 2022/4/14 11:03

import jieba
import re

import pandas

# 夏洛特烦恼台词
filepath1 = 'C://PY//bleat//sample//Data//夏洛特烦恼'
dict = []

# 创建字典
with open(filepath1, 'r', encoding='utf-8') as sourceFile:
    # 按行读数据
    lines = sourceFile.readline()
    for line in lines:
        # 去掉文本中的空格
        line1 = line.replace(' ', '')
        # 只保留中英文
        pattern = re.compile("[^\u4e00-\u9fa5^a-z^A-Z^0-9]")
        # 吧文本中匹配到的字符替换成空字符
        line2 = re.sub(pattern, '', line1)
        seg = jieba.lcut(line2, cut_all=False)  # 结巴分词对处理过的文本进行分词
        for i in range(len(seg)):
            if seg[i] not in dict:
                dict.append(seg[i])
    print(len(dict))
    print(dict)
# 创建字典
with open(filepath1, 'r', encoding='utf-8') as sourceFile:
    lines = sourceFile.readline()  # 按行读取文件
    for line in lines:
        ##去掉文本中的空格
        line1 = line.replace(' ', '')
        ##只保留中英文
        pattern = re.compile("[^\u4e00-\u9fa5^a-z^A-Z^0-9]")

        # 把文本中匹配到的字符替换成空字符

        line2 = re.sub(pattern, '', line1)
        seg = jieba.lcut(line2, cut_all=False)
        for i in range(len(seg)):
            if seg[i] not in dict:
                dict.append(seg[i])
    print(len(dict))
    print(dict)
# 前向最大匹配算法
def forword_Match(text, Dict):
    word_list = []
    # 初始位置
    pi = 0
    # 找出字典中的最长的词的长度
    m = max([len(word) for word in Dict])
    while pi != len(text):
        n = len(text[pi:])
        if n < m:
            m = n
            # 从当前 pi 起取 m 个汉字作为词
        for index in range(m, 0, -1):
            if text[pi:pi + index] in Dict:

                word_list.append(text[pi:pi + index])

                # 根据词的长度修改指针pi
                pi = pi + index
                break
    print('/'.join(word_list), len(word_list))

    # 前向最大匹配算法
    def forword_Match1(text, Dict):
        word_list = []
        # 初始位置
        pi = 0
        # 找出字典中的最长的  词的长度
        m = max([len(word) for word in Dict])
        while pi != len(text):
            n = len(text[pi:])
            if n < m:
                m = n
                # 从当前 pi 取 m 个汉字作为词
            for index in range(m, 0, -1):
                if text[pi:pi + index] in Dict:
                    word_list.append(text[pi:pi + index])
                    # 根据词的长度修改指针pi
                    pi = pi + index
                    break
        print('/'.join(word_list), len(word_list))
